package rafael.bot;

import java.util.Random;

public class QModel extends Model{

	private double[][] Q;
	private double alpha, gama;
	private int memory;
	private Random random;
	public QModel(double alpha, double gama, int memory) {
		this.alpha = alpha;
		this.gama = gama;
		this.memory = memory;
		random = new Random();
	}
	private int getState(int myMove, int itsMove) {
		return (myMove-1)*3+(itsMove-1);
	}
	@Override
	public void init(int movs) {
		//state is (myLastMove,itsLastMove) represented as 1-9
		Q = new double[9][4];
	}

	private double score(int myMove, int itsMove) {
		if(bestAnswer(itsMove) == myMove) {
			return 100;
		}else if(bestAnswer(myMove) == itsMove) {
			return -100;
		}
		return -1;
	}
	
	@Override
	public void learn(int[] input, int output, double learningRate) {
		if(input[1] == 0 || input[memory+1] == 0){
			return;
		}
		
		int itsMove = input[memory + 1];//estado = movimento anterior do inimigo
		int myMove = input[1];//acao = meu movimento de resposta
		
		int s = getState(myMove,itsMove);//estado representando a configuracao
		int a = input[0];
		//novo estado = movimento mais recente do inimigo
		int s_ = getState(input[0],input[memory]);
		
		//recompensa dada a acao e estado anterior
		double r = score(input[0],input[memory]);
		double maxAs_ = Double.NEGATIVE_INFINITY;
		for(int i = 0; i < Q[s_].length; i++) {
			if(Q[s_][i] > maxAs_) {
				maxAs_ = Q[s_][i];
			}
		}
		//novo valor do estado
		Q[s][a] = Q[s][a] + alpha*(r + gama*maxAs_ - Q[s][a]);
	}

	@Override
	public int predictMove(int[] input) {
		//input = myMoves[memory] | itsMoves[memory]
		if(input[0] == 0) {
			return 1;
		}
		
		int s = getState(input[0],input[memory]);//its most recent move
		int action = 1;
		double[] q = VectorUtils.pDist(Q[s]);
		double ac = 0;
		double u = random.nextDouble();
		for(int i = 0; i < q.length; i++) {
			ac += q[i];
			if(u < ac) {
				action = i;
				break;
			}
		}
		int prediction = 1;
		for(int j = 1; j <= 3; j++) {
			if(bestAnswer(j) == action) {
				prediction = j;
			}
		}
		return prediction;
	}

}
